import pandas as pd
import numpy as np
np.random.seed(2022-11-8) # for reproducibility
n_target = 10000
target_df = pd.DataFrame(
{
"gender": np.random.choice(["Male", "Female"], size = n_target, replace = True, p= [.5,.5]),
"age_group": np.random.choice(["18-24","25-34", "35-44", "45+"], size = n_target, replace = True, p= [.20, .30, .30,.20]),
"income": np.random.normal(3, 2, size = n_target)**2,
# "unrelated_variable": np.random.uniform(size = n_target),
"id": (np.array(range(n_target)) + 100000).astype(str),
# "weight": np.random.uniform(size = n_target) + 0.5,
}
)
target_df.happiness = np.random.normal(50, 10, size = n_target) + \
np.where(target_df.gender == "Female", 1, 0) * np.random.normal(5, 2, size = n_target)
# We also have missing values in gender
target_df.loc[3:900, "gender"] = np.nan
n_sample = 1000
sample_df = pd.DataFrame(
{
"gender": np.random.choice(["Male", "Female"], size = n_sample, replace = True, p= [.7,.3]),
"age_group": np.random.choice(["18-24","25-34", "35-44", "45+"], size = n_sample, replace = True, p= [.50, .30, .15,.05]),
"income": np.random.normal(2, 1.5, size = n_sample)**2,
# "unrelated_variable": np.random.uniform(size = n_sample),
"id": (np.array(range(n_sample))).astype(str),
# "weight": np.random.uniform(size = n_sample) + 0.5,
}
)
# females are happier
# older people are happier
# people with higher income are happeir
sample_df["happiness"] = np.random.normal(40, 10, size = n_sample) + \
np.where(sample_df.gender == "Female", 1, 0) * np.random.normal(20, 1, size = n_sample) + \
np.where(sample_df.age_group == "35-44", 1, 0) * np.random.normal(5, 1, size = n_sample) + \
np.where(sample_df.age_group == "45+", 1, 0) * np.random.normal(20, 1, size = n_sample) + \
np.random.normal((np.random.normal(3, 2, size = n_sample)**2) / 20, 1, size = n_sample)
# Truncate for max to be 100
sample_df["happiness"] = np.where(sample_df["happiness"] < 100, sample_df["happiness"], 100)
# We also have missing values in gender
sample_df.loc[3:90, "gender"] = np.nan
